﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.Collections;
using System.Net;
using System.IO;
using System.Threading;

namespace week5
{
    public partial class Form1 : Form
    {
        static int emails = 0;
        static int count = 0;
        static bool t = false;
        private Hashtable urls = new Hashtable();
        private Hashtable emailadr = new Hashtable();
        public Form1()
        {
            InitializeComponent(); 
        }

        //初始化界面
        private void Form1_Load(object sender, EventArgs e)
        {
            try
            {
                
                listView1.Columns.Add("邮箱地址", 250);
                listView1.Columns.Add("URL", 300);
                listView1.Columns.Add("编号", 50);
                listView1.View = View.Details;
                listBox1.Items.Add("已爬取的网址：");
            }
            catch { }
        }

        //显示搜索网页
        private void button1_Click(object sender, EventArgs e)
        {
            string key = textBox1.Text;
            webBrowser1.ScriptErrorsSuppressed = true;
            webBrowser1.Navigate("https://www.baidu.com/s?wd=" + key);//显示网页
            
        }

        //爬行程序
        public void search()
        {
            try
            {

                while(true)
                {
              
                    string current = null;
                    foreach(string url in urls.Keys)//选取没有爬取过的网址
                    {
                        if ((bool)urls[url]) continue;
                        current = url;
                    }
                    if (current == null  )
                    {
                        t = false;
                        return;
                    }
                    count++;
                   
                    string html = Craw(current);//获取网页html源代码

                    listBox1.Items.Add(count.ToString() + ":" + current);
                    Console.WriteLine(count.ToString() + ":" + current + "=========================================================================================");
                    urls[current] = true;
                    if (html == null) continue;

                    Findemails(html, current) ;//查找其中的邮箱地址
                    if (emails >= 50)
                    {
                        t = false;
                        return;
                    }

                    if (!Findurl(html)) continue;//查找其中的网址
                    if (count == 0 && !t) t = true;
                }
            }
            catch { }
            
        }


        //查找网址
        private bool Findurl(string html)
        {
            try
            {
                string strRef = @"(href|HREF)=[""][^""%]+[""]"; //网址正则表达式
                MatchCollection matches = new Regex(strRef).Matches(html);//匹配
                foreach (Match match in matches)
                {
                    strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\'', '#', ' ', '>');
                    if (strRef.Length == 0) continue;
                    if (urls[strRef] == null) urls[strRef] = false;
                }
                return true;
            }
            catch { return false; }
        }

        //查找邮箱地址
        private void Findemails(string html,string current)
        {
            try
            {
                string email = @"[a-zA-Z0-9_\-\.]+@[a-zA-Z0-9]+\.[\.a-zA-Z0-9]+";//邮箱地址正则表达式
                MatchCollection matches1 = new Regex(email).Matches(html);//匹配
                foreach (Match match in matches1)
                {
                    if (emails >= 50) return;
                    if (emailadr[match.Value] == null) emailadr[match.Value] = false;
                    if ((bool)emailadr[match.Value]) continue;
                    emails++;
                    Console.WriteLine(match.Value + "-------------------------------------------------------------------");
                    ListViewItem lvi = listView1.Items.Add(match.Value);
                    lvi.SubItems.Add(current);
                    lvi.SubItems.Add(emails.ToString());
                    emailadr[match.Value] = true;
                }
     
            }
            catch {  }
        }

        //获取网页源码
        private string Craw(string current)
        {
            try
            {
                string html;
                //创建web服务器的请求
                HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create(current);
                httpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36";


                //获得Web服务器的响应
                HttpWebResponse httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse();

                //查看网页的响应，如果StatusCode!=200就返回null！
                if (httpWebResponse.StatusCode != HttpStatusCode.OK)
                {
                    return null;
                }

                //获得网页的返回流
                Stream wangyeyuanma = httpWebResponse.GetResponseStream();
                StreamReader streamReader = new StreamReader(wangyeyuanma);

                //读取流中的数据并转化为string
                html = streamReader.ReadToEnd();
                Console.WriteLine(html);
                return html;
            }
            catch { return null; }
        }

        //开始爬行按钮
        private void button2_Click(object sender, EventArgs e)
        {
            try
            {
                urls.Clear();
                emailadr.Clear();
                count = 0;
                emails = 0;
                listBox1.Items.Clear();
                string starturl = webBrowser1.Url.ToString();//获取当前页面网址
                urls.Add(starturl, false);
                search();
                if (t)//多线程
                {
                    Thread t2 = new Thread(() => search());
                    t2.Start();
                }
                Console.WriteLine("爬取结束!!");
            }
            catch { return; }
        }
    }
}
